#!/usr/bin/python
# -*- coding: utf-8 -*-
# Description: 获取ftpip
# Create at 2008-11-3 下午08:31:57
# Author: MK2[fengmk2@gmail.com]

"""docstrings
"""

import os
import sys
from web.spider import Spider
from ftpspider import DEFAULT_FTPIP_PATTERN, check_ftpip_is_repeat

class FtpIPSpider(object):
    
    def __init__(self, url, templates, result_path):
        self.result_path = result_path
        last_url = self.get_last_grab_url()
        if last_url:
            url = last_url
        self.spider = Spider(url, templates)
        self._f = None
        
    def save(self, ftpip):
        if self._f is None:
            self._f = open(self.result_path, 'ab')
        self._f.write(ftpip + '\r\n')
        self._f.flush()
    
    def save_grab_url(self, url):
        """保存当前抓取的web url"""
        f = open(self.result_path + '_last.url', 'wb')
        try:
            f.write(url)
        finally:
            f.close()
        
    def get_last_grab_url(self):
        """获取最后抓起的url，作为起点"""
        if not os.path.exists(self.result_path + '_last.url'):
            return None
        f = open(self.result_path + '_last.url', 'rb')
        try:
            return f.read()
        finally:
            f.close()
    
    def run(self):
        for level, url, parenturl, response, \
                content, downurls, softinfo in self.spider.walk():
            self.save_grab_url(url)
            for ftpurl in downurls:
                if not check_ftpip_is_repeat(ftpurl):
                    self.save(ftpurl)
        if self._f is not None:
            self._f.close()
            
    def __del__(self):
        if self._f is not None:
            self._f.close()
        
        
if __name__ == '__main__':
    no = sys.argv[1]
#    mmnt_templates = [{
#            'same_level': ur"""
#            <A[^<>]*?HREF="(/int/get[^<>]*?st=exe[^<>]*?)"[^<>]*?>(?:(?:Skip)|(?:Next))[^<>]*?</A>
#            """,
#            'downurl': DEFAULT_FTPIP_PATTERN,
#        }, {}]
#    mmnt_url = 'http://www.mmnt.ru/int/get?in=f&st=exe&ot=2473061'
#    mmnt_savepath = '/home/mk2/ftp_ips_%s.lst' % no
#    s = FtpIPSpider(templates=mmnt_templates,
#               url=mmnt_url,
#               result_path=mmnt_savepath)
#    s.run()
    
    while True:
        try:
            sf_templates = [{
                    'same_level': ur"""
                    <a[^<>]*?href="(search\.php\?String=\.exe[^<>]*?)">[^<>]*?</a>
                    """,
                    'downurl': DEFAULT_FTPIP_PATTERN,
                }, {}]
            sf_url = 'http://sf.hit.edu.cn/search.php?String=.exe&Site=&UseVip=&Type=&Dl=&SortType=&Page=2'
            sf_savepath = '/home/mk2/ftp_ips_%s.lst' % no
            s = FtpIPSpider(templates=sf_templates,
                       url=sf_url,
                       result_path=sf_savepath)
            s.run()
        except Exception, e:
            print e